#coding:utf-8
# write  by  zhou
import gevent
from gevent.monkey import  patch_all;patch_all()
import json
import random
import time
import redis
import requests
from celery import Celery
from gcpy_utils.simple_config import parse_config
import simplejson
import sys
import ssl
context = ssl._create_unverified_context()
reload(sys)
sys.setdefaultencoding("utf-8")

config_dict = parse_config("../../")
# celery 配置
app = Celery()
app.conf.task_ignore_result = True
app.conf.task_queue_max_priority = 255
app.conf.broker_url = config_dict["starchain_celery_service"]["broker_redis_url"]
app.conf.task_default_queue = "pagespeed"
celery_redis_conn = redis.Redis.from_url(app.conf.broker_url)
import random

def page_speed_log(log):

    "page_speed的日志"
    celery_redis_conn.lpush("page_speed_log", simplejson.dumps({"log": log, "addtime": int(time.time())}))
    celery_redis_conn.ltrim("page_speed_log", 0, 500)


@app.task(ignore_result=True)
def celery_page_speeds(url,pageid=None,callback_url=None,duration=None,timeout=5,**kwargs):
    '''
         page_speed  请求计算
         url： 要分析的URL
         callback_url:回调的地址
         duration : 间隔时间
	     pagespeed 文档参数
	     url  要获取和分析的url
	     可选参数
	     filter_third_party_resources               是否过滤掉但三方资源false 或true
	     locale                                     本地格式化语言   zh_CN
         rule                                       要运行的规则    没有给出默认全选
         screenshot                                 指示是否包含截图二进制数据应包括（缺省：false）
         snapshots                                  指示是否包含快照图像的二进制数据应包括（缺省：false）
         strategy                                   要使用的分析策略（桌面或移动），而桌面是默认的 Acceptable值是：“ desktop”：获取并分析桌面浏览器的URL“ mobile”：获取并分析移动设备的URL
         utm_source                                 分析的广告系列来源
         utm_campaign                               分析的广告系列名称

    '''
    print 'hello '
    retries = 50 if kwargs.get("retries") == None else kwargs.get("retries")
    if duration:
       duration = max(30, int(duration), timeout)
    try:
       kwargs.pop("retries")
    except:
        pass
    try:
        with gevent.Timeout(timeout,'timeout') as t:  # 超时返回值获取
            with open( "./success_sokcs4.txt","r") as f:
                ipList = f.read().split("\n")
                proxy = random.choice(ipList)
                proxies = {'http': proxy, 'https': proxy}
                    # ipList =[{'http': 'socks5://81.30.132.30:31446','https': 'socks5://81.30.132.30:31446'},
                    #          {'http': 'socks4://103.242.51.58:4145','https': 'socks4://103.242.51.58:41ps45'},
                    #          {'http': 'socks4://176.118.48.150:4145', 'https': 'socks4://176.118.48.150:4145'},
                    #          {'http': 'socks4://101.255.90.110:61221', 'https': 'socks4://101.255.90.110:61221'},
                    #          {'http': 'socks4://103.57.80.48:4145', 'https': 'socks4://103.57.80.48:4145'},
                    #          ]
                        # proxies = {'http': "socks5://admin:gc895316@199.247.18.222:1080","https": "socks5://admin:gc895316@199.247.18.222:1080"}
                rep = requests.get("https://www.googleapis.com/pagespeedonline/v4/runPagespeed", {
                   'url': str(url),
                   'locale': 'zh_CN',
                   'filter_third_party_resources': False,
                   'strategy': 'desktop'
                }, proxies=proxies)
                assert rep.status_code == 200, 'rep status code is %s' % (rep.status_code)
    except (Exception,BaseException,gevent.Timeout) as e:
        print e
        print url,pageid,
        if 'rep status code ' in str(e):
           page_speed_log("get %s exception %s \n `pageid`:%s \n `response.text`: %s " % (url,str(e),pageid,rep.text))
        else:
           page_speed_log('get %s exception %s \n `pageid`:%s\n ' % (url,str(e),pageid))
        if retries > 0:
            retries -= 1
            kwargs["retries"] = retries
            app.send_task("pagespeed.celery_page_speeds",
                         kwargs={'url': url,
                                 'pageid':pageid,
                                 'callback_url': callback_url,
                                 'duration': duration,
                                 'timeout': timeout
                                 },
                         queue="pagespeed")
    else:
        page_speed_log('分析 %s successful `response.text`: %s' % (url,rep.text))
        if callback_url:
            app.send_task('async_requests.post',
                         kwargs={'url': callback_url,
                                 'data': {'data': rep.text,'pageid':pageid},
                                 'duration': duration,
                                 'timeout': timeout},
                         queue='async_requests')

            print rep.text
            print [proxies,url]


def schedule_loop(*args,**kwargs):

    '''
    page_speed 调度器--调度对pagespeed 的分析计算
    :param args:
    :param kwargs:
    :return:
    '''

    gevent.sleep(5)
    print 'start_schedule_loops...'
    while 1:
        gevent.sleep(max(2,random.random() * 6))    # 随机停某些秒
        print 'loops........*.....。'
        now_timestamp = int(time.time())
        pageids = celery_redis_conn.zrevrangebyscore("page_speed",now_timestamp,0,start=0,num=30)    # 返回有序集合指定分数区间内的成员
        if not pageids:
            continue
        random.shuffle(pageids)   # 将序列的所有元素随机排序，shuffle 不能直接访问，需要random 静态方法调用该方法
        pageids_configs = dict(zip(pageids,celery_redis_conn.hmget("page_speed_config",pageids)))

        for pageid in pageids:
            try:
                config = json.loads(pageids_configs[pageid])
                pageid = config["pageid"]
                url = config["url"]
                callback_url = config["callback_url"]
                duration = config['duration']
                timeout = config['timeout']
            except Exception as e:
                pass
            else:
                # 防止多个 worker 处理同一个任务
                if celery_redis_conn.incr("_page_%s_flag" % pageid, 1) == 1:
                    celery_redis_conn.zincrby("page_speed", pageid, duration)
                celery_redis_conn.delete("_page_%s_flag" % pageid)
                app.send_task("pagespeed.celery_page_speeds",
                              kwargs={'url':url,
                                      'pageid':pageid,
                                      'callback_url':callback_url,
                                      'duration':duration,
                                      'timeout':timeout
                                      },
                              queue ="pagespeed")
                print [url, pageid]

gevent.spawn(schedule_loop)



